/*
 * ThaiSeg.java
 *
 * Created on 04 March 2007, 18:18
 */
package net.siit.ling;

import java.io.*;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.speedblue.util.*;

import net.siit.datatype.AnnotatedRange;
import net.siit.datatype.Range;

/**
 *
 * @author art
 */
public class ThaiSeg {
    private int maxDictLength = 0;
    private int minDictLength = 32000;

    // tokenization strategies
    static public final int LONGEST_MATCH = 1;
    static public final int MAXIMUM_MATCH = 2;

    static public final int TOKEN_LABEL_OTHERS = 0;
    static public final int TOKEN_LABEL_WHITESPACE = 1;
    static public final int TOKEN_LABEL_PUNCTUATION = 2;
    static public final int TOKEN_LABEL_SYMBOL = 3;
    static public final int TOKEN_LABEL_ARABIC_DIGITS = 4;
    static public final int TOKEN_LABEL_THAI_DIGITS   = 5;
    static public final int TOKEN_LABEL_THAI_NON_DICTIONARY_WORD = 6;
    static public final int TOKEN_LABEL_THAI_DICTIONARY_WORD     = 7;

    static public final int CLUSTER_LABEL_OTHERS        = 10;
    static public final int CLUSTER_LABEL_WHITESPACE    = 11;
    static public final int CLUSTER_LABEL_SYMBOL        = 13;
    static public final int CLUSTER_LABEL_ARABIC_DIGITS = 14;
    static public final int CLUSTER_LABEL_THAI_DIGITS   = 15;
    static public final int CLUSTER_LABEL_TCC           = 16;
    static public final int CLUSTER_LABEL_ATCC          = 17;
    static public final int CLUSTER_LABEL_BTCC          = 18;
    
    static public final String LANG_ENG = "eng"; // English
    static public final String LANG_THA = "tha"; // Thai
    static public final String LANG_MIS = "mis"; // uncoded langauges
    static public final String LANG_MUL = "mul"; // multiple languages
    static public final String LANG_UND = "und"; // undeterminded
    static public final String LANG_ZXX = "zxx"; // no linguistic content
        
    static public final URL DEFAULT_DICTIONARY_URL = 
            ThaiSeg.class.getResource("/net/siit/ling/resources/thai.dic");
    private Trie dict;
    private TCCSeg tccSeg;
    
    public ThaiSeg() throws Exception {
        this(DEFAULT_DICTIONARY_URL, "UTF-8");  // use default dictionary
    }

    /** Creates a new instance of ThaiSeg
     * @throws java.io.FileNotFoundException
     * @throws java.io.UnsupportedEncodingException
     * @throws java.io.IOException
     * @throws java.lang.Exception
     */
    public ThaiSeg(URL dictUrl, String dictCharset) throws FileNotFoundException, UnsupportedEncodingException, IOException, Exception {
        InputStream dictIS = new FileInputStream(dictUrl.getPath());
        BufferedReader dictReader = new BufferedReader(new InputStreamReader(dictIS, dictCharset));
        dict = new Trie(1000);
        tccSeg = new TCCSeg();

        String line = null;
        int len = -1;
        long size = 0;
        while ((line = dictReader.readLine()) != null) {
            try {
                len = line.length();
                dict.addEntry(line, len);
                size++;
                if (maxDictLength < len) {
                    maxDictLength = len;
                }
                if (minDictLength > len) {
                    minDictLength = len;
                }
            } catch (Exception ex) {
                Logger.getLogger("global").log(Level.SEVERE, null, ex);
                throw new Exception("While creating trie: ", ex);
            }
        }

        dictReader.close();
        dictIS.close();

//        System.err.println("max  = " + maxDictLength);
//        System.err.println("min  = " + minDictLength);
//        System.err.println("size = " + size);
    }

    /**
     *
     * @param text
     * @param strategy
     * @return
     * @throws java.lang.Exception
     */
    public List<AnnotatedRange<Integer>> tokenize(String text, int strategy) throws Exception {
        if (strategy == LONGEST_MATCH) {
            return tokenizeLongestMatch(text);
        } else if (strategy == MAXIMUM_MATCH) {
            return tokenizeMaximumMatch(text);
        } else {
            throw new IllegalArgumentException("Unknown tokenization strategy");
        }
    }

    /**
     * Tokenizes text using "longest matching - left to right" strategy:
     * 1) choose the match that closest to the string starting point
     * 2) if matches share a starting point, chose the longest match
     *    (logic embedded into Range#compareTo)
     * 3) start from left to right
     *
     * @param text
     * @return list of token ranges
     */
    public List<AnnotatedRange<Integer>> tokenizeLongestMatch(String text) {
        int len = text.length();
        List<AnnotatedRange<Integer>> ranges = new ArrayList<AnnotatedRange<Integer>>();

        if (len < minDictLength) {
            // instead of returning an empty list,
            // return a list with one range containing non-word,
            // in which that non-word is the whole original input text
            ranges.add(new AnnotatedRange<Integer>(0, len - 1, TOKEN_LABEL_THAI_NON_DICTIONARY_WORD));
            return ranges;
        }

        int start = 0;
        int end = start + minDictLength;  // end will grows from start+minDictLength to start+maxDictLength
        int cStart;  // candidate's start
        int cEnd;    // candidate's end
        int nwStart; // non-word's start
        int nwEnd;   // non-word's end

        nwStart = -1;
        nwEnd = -1;
        // go left to right, until the end of string
        while (end <= len) {
            cStart = -1;
            cEnd = -1;
            // go from left to right,
            // until the window is exceeded the maximum dictionary word length
            while ((end <= len) && (end < start + maxDictLength)) {
                end = nextValidBreakRight(end, text);
                // check if this candidate is in the dictionary or not
                if (dict.getEntry(text.substring(start, end)) > -1) {
                    // cStart and cEnd will remember the most recent range
                    // that match dictionary word
                    cStart = start;
                    cEnd = end;
                }
                end++; // shift window's end further one character, to the right
            }

            // cStart equals -1 means not found in dictionary.
            // group contiguous non-space characters together as one non-word
            if (cStart == -1) {
                // Mark continuous non-space characters as one non-word range
                // Example:
                //   dictionary     : กิน
                //   input          : ก_กกกกิน
                //   expected output: ก/nw _/nw กกก/nw กิน/w
                //   wrong output   : ก/nw _/nw ก/nw ก/nw ก/nw กิน/w

                // if "stop-character" found, add itself and new non-word range, if any
                if (isStopCharacter(text.codePointAt(start))) {
                    // if non-word range is not empty
                    if (nwEnd != nwStart) {
                        // add non-word range
                        ranges.add(new AnnotatedRange<Integer>(nwStart, nwEnd, TOKEN_LABEL_THAI_NON_DICTIONARY_WORD)); // non-word
                        // reset non-word range
                        nwStart = -1;
                        nwEnd = -1;
                    }
                    // add a "stop-character" as well
                    ranges.add(new AnnotatedRange<Integer>(start, start + 1, TOKEN_LABEL_PUNCTUATION)); // space
                } else {
                    if (nwStart == -1) {
                        // start new non-word range
                        nwStart = start;
                        nwEnd = nwStart + 1;
                    } else {
                        // enlarge non-word range to the right
                        nwEnd++;
                    }
                }
                start++; // shift window's start further one character, to the right
            } else {
                // if non-word range is not empty, add it first (before add new word range)
                if (nwEnd != nwStart) {
                    // add non-word range
                    ranges.add(new AnnotatedRange<Integer>(nwStart, nwEnd, TOKEN_LABEL_THAI_NON_DICTIONARY_WORD)); // non-word
                    // reset non-word range
                    nwStart = -1;
                    nwEnd = -1;
                }
                // add new word range
                ranges.add(new AnnotatedRange<Integer>(cStart, cEnd, TOKEN_LABEL_THAI_DICTIONARY_WORD));
                start = cEnd;
            }

            end = start + minDictLength;
            //end = nextValidBreakRight(end, text);
        }
        
        // if non-word range is not empty
        if (nwEnd != nwStart) {
            // add non-word range
            ranges.add(new AnnotatedRange<Integer>(nwStart, nwEnd, TOKEN_LABEL_THAI_NON_DICTIONARY_WORD)); // non-word
            // no need to reset non-word range, as this is the last line
        }

        Collections.sort(ranges);
        return ranges; // don't sort this! ... well, why?
    }

    /**
     * Tokenizes text using "longest matching - left to right" strategy:
     * 0) clusterizes string to TCCs
     * 1) choose the match that closest to the string starting point
     * 2) if matches share a starting point, chose the longest match
     *    (logic embedded into Range#compareTo)
     * 3) start from left to right
     *
     * @param text
     * @return list of token ranges
     */
    public List<AnnotatedRange<Integer>> tokenizeLongestMatch(List<AnnotatedRange<Integer>> clusters, String text) {
        int len = clusters.size();
        List<AnnotatedRange<Integer>> ranges = new ArrayList<AnnotatedRange<Integer>>();
        
        if (len <= 0) {
            return ranges;
        }

        int start = 0;
        int end = 0;
        int cStart;  // candidate's start
        int cEnd;    // candidate's end

        // go left to right, until the end of string
        while (end < len) {
            cStart = -1;
            cEnd = -1;
            // go from left to right,
            // until the window is exceeded the maximum dictionary word length
            while ((end < len) && (clusters.get(end).getEnd() < clusters.get(start).getStart() + maxDictLength)) {
                // check if this candidate is in the dictionary or not
                // NOTE: there should be a possible short cut here
                // NOTE: the trie should be able to smartly answer the query
                // NOTE: question, "do you have 'abcdef'?"
                // NOTE: answer, "we have only 'abcde', and just that, no 'abcdeXXWHATEVERXX'"
                if (dict.getEntry(text.substring(clusters.get(start).getStart(), clusters.get(end).getEnd())) > -1) {
                    // [cStart,cEnd] = most recent range that matched word in dictionary
                    cStart = start;
                    cEnd = end;
                }
                end++; // shift window's end further one character, to the right
            }
            
            // cStart equals -1 means not found in dictionary.
            if (cStart == -1) {
                ranges.add(clusters.get(start));
                start++; // shift window's start further one tcc, to the right
            } else {
                ranges.add(new AnnotatedRange<Integer>(clusters.get(cStart).getStart(), clusters.get(cEnd).getEnd(), TOKEN_LABEL_THAI_DICTIONARY_WORD));
                start = cEnd + 1; // tcc-based range is different from character-based range
            }

            end = start;
        }
        
        Collections.sort(ranges);
        return ranges; // don't sort this! ... well, why?
    }
    
    /**
     * Tokenizes text using "longest matching - right to left" strategy:
     * 1) choose the match that closest to the string starting point
     * 2) if matches share a starting point, chose the longest match
     *    (logic embedded into Range#compareTo)
     * 3) start from right to left
     *
     * @param text
     * @return list of token ranges
     */
    public List<AnnotatedRange<Integer>> tokenizeLongestMatchRL(String text) {
        int len = text.length();
        List<AnnotatedRange<Integer>> ranges = new ArrayList<AnnotatedRange<Integer>>();
        
        if (len < minDictLength) {
            // instead of returning an empty list,
            // return a list with one range containing non-word,
            // in which that non-word is the whole original input text
            ranges.add(new AnnotatedRange<Integer>(0, len - 1, TOKEN_LABEL_THAI_NON_DICTIONARY_WORD));
            return ranges;
        }

        int start = len-minDictLength; // start will shrinks from len-minDictLength to 0
        int end = len;
        int fixedStart;
        int cStart;  // candidate's start
        int cEnd;    // candidate's end
        int nwStart; // non-word's start
        int nwEnd;   // non-word's end

        nwStart = -1;
        nwEnd = -1;
        // go from right to left, until the start of string
        while (start >= 0) {
            cStart = -1;
            cEnd = -1;
            
            fixedStart = start; // remember start
            // go from right to left,
            // until the window is exceeded the maximum dictionary word length
            while ((start >= 0) && (end < start + maxDictLength)) {
                // specifically for Thai:
                start = nextValidBreakLeft(start, text);
                if (dict.getEntry(text.substring(start, end)) > -1) {
                    // cStart and cEnd will remember the most recent range
                    // that match dictionary word
                    cStart = start;
                    cEnd = end;
                    
                    //System.out.println("found: " + text.substring(start, end));
                }
                start--; // shift window's start further one character, to the left
            }
            start = fixedStart; // restore start

            // cStart equals -1 means not found in dictionary.
            // group contiguous non-space characters together as one non-word
            if ((cStart == -1)) {
                // Mark continuous non-space characters as one non-word range
                // Example:
                //   dictionary     : กิน
                //   input          : ก_กกกกิน
                //   expected output: ก/nw _/nw กกก/nw กิน/w
                //   wrong output   : ก/nw _/nw ก/nw ก/nw ก/nw กิน/w

                // if "stop-character" found, may add new non-word range
                if (isStopCharacter(text.codePointAt(start))) {
                    // if non-word range is not empty
                    if (nwEnd != nwStart) {
                        // add non-word range
                        ranges.add(new AnnotatedRange<Integer>(nwStart, nwEnd, TOKEN_LABEL_THAI_NON_DICTIONARY_WORD)); // non-word
                        // reset non-word range
                        nwStart = -1;
                        nwEnd = -1;
                    }
                    // add a "stop-character" as well
                    ranges.add(new AnnotatedRange<Integer>(start, start + 1, TOKEN_LABEL_PUNCTUATION)); // space
                } else {
                    if (nwStart == -1) {
                        // start new non-word range
                        nwStart = start;
                        nwEnd = nwStart + 1;
                    } else {
                        // enlarge non-word range to the left
                        nwStart--;
                    }
                }
                start--; // shift window's end further one character, to the left
                end = start + minDictLength;
                //end = nextValidBreakRight(end, text);
            } else {
                // found word in dictionary
                
                // if non-word range is not empty
                if (nwEnd != nwStart) {
                    // add non-word range
                    ranges.add(new AnnotatedRange<Integer>(nwStart, nwEnd, TOKEN_LABEL_THAI_NON_DICTIONARY_WORD)); // non-word
                    // reset non-word range
                    nwStart = -1;
                    nwEnd = -1;
                }
                // add new word range
                ranges.add(new AnnotatedRange<Integer>(cStart, cEnd, TOKEN_LABEL_THAI_DICTIONARY_WORD));
                start = cStart - minDictLength;
                end   = cStart;
                //end   = nextValidBreakRight(end, text);
            }
        }
        
        // if non-word range is not empty
        if (nwEnd != nwStart) {
            // add non-word range
            ranges.add(new AnnotatedRange<Integer>(nwStart, nwEnd, TOKEN_LABEL_THAI_NON_DICTIONARY_WORD)); // non-word
            // no need to reset non-word range, as this is the last line
        }

        Collections.sort(ranges);
        return ranges; // don't sort this! ... well, why?
    }

    /**
     * Tokenizes text using "maximum matching" strategy,
     * choose a set of tokens that has the highest numbers of tokens.
     *
     * @param text
     * @return
     */
    public List<AnnotatedRange<Integer>> tokenizeMaximumMatch(String text) {
        int len = text.length();
        List<AnnotatedRange<Integer>> ranges = new ArrayList<AnnotatedRange<Integer>>();

        if (len < minDictLength) {
            return ranges;
        }

        //TODO

        return ranges;
    }
    
    /**
     * Only experimental
     * 
     * For serious usage, please consider ICU4J.
     * 
     * @param text
     * @return
     */
    public static List<AnnotatedRange<String>> labelLanguage(String text) {
        List<AnnotatedRange<String>> segments
                = new ArrayList<AnnotatedRange<String>>();
        char[] chs = text.toCharArray();

        boolean prevMode = ((chs[0] >= '\u0e00') && (chs[0] <= '\u0eff'));
        boolean currMode;
        int prevStartPos = 0;
        for (int i = 1; i < chs.length; i++) {
            currMode = ((chs[i] >= '\u0e00') && (chs[i] <= '\u0eff')); // does it Thai char?
            if (currMode != prevMode) {
                AnnotatedRange<String> r = new AnnotatedRange<String>();
                r.setStart(prevStartPos);
                r.setEnd(i);
                if (prevMode) {
                    r.setLabel(LANG_THA);
                } else {
                    r.setLabel(LANG_UND);
                }
                segments.add(r);
                prevStartPos = i;
            }
            prevMode = currMode;
        }
        if (prevStartPos != chs.length) {
             AnnotatedRange<String> r = new AnnotatedRange<String>();
                r.setStart(prevStartPos);
                r.setEnd(chs.length);
                if (prevMode) {
                    r.setLabel(LANG_THA);
                } else {
                    r.setLabel(LANG_UND);
                }
                segments.add(r);
        }
        
        return segments;
    }

    public static int nextValidBreakRight(int initBreak, String text) {
        int len = text.length();
        if (initBreak >= len)
            return initBreak;
        
        char check;
        boolean changed = true;
        
        while ((changed == true) && (initBreak < len)) {
            changed = false;

            // specifically for Thai:
            // if current char is "leading character",
            // e.g. leading vowels,
            // don't cut it here (it need some following chars)
            if (initBreak > 0) {
                check = text.charAt(initBreak-1);
                if (check >= 0x0e40 && check <= 0x0e44) {  // sara e - sara ai maimalai
                    initBreak++;
                    changed = true;
                }
            }
            
            // if next char (on the right) is "following character",
            // e.g. tone marks, some vowels,
            // don't cut it here
            if (initBreak < len) {
                check = text.charAt(initBreak);
                if ((check >= 0x0e30 && check <= 0x0e3a) || // sara a - phinthu
                        (check == 0x0e45) || // lakkhang yao
                        (check >= 0x0e47 && check <= 0x0e4e)) {  // mai taikhu - yamakkan
                        initBreak++;
                        changed = true;
                }
            }
        }
        
        return initBreak;
    }

    public static int nextValidBreakLeft(int initBreak, String text) {
        if (initBreak <= 0)
            return initBreak;

        char check;
        boolean changed = true;
        
        while ((changed == true) && (initBreak > 0)) {
            changed = false;
            
            // if current char is "following character",
            // e.g. tonemark, following vowels,
            // don't cut it here (a word can't begin with this char)
            check = text.charAt(initBreak);
            if ((check >= 0x0e30 && check <= 0x0e3a) || // sara a - phinthu
                    (check == 0x0e45) || // lakkhang yao
                    (check >= 0x0e47 && check <= 0x0e4e)) {  // mai taikhu - yamakkan
                initBreak--;
                changed = true;
            }
            
            // if next char (on the left) is "leading character",
            // e.g. leading vowels,
            // don't cut it here
            if (initBreak > 0) {
                check = text.charAt(initBreak - 1);
                if (check >= 0x0e40 && check <= 0x0e44) {  // sara e - sara ai maimalai
                    initBreak--;
                    changed = true;
                }
            }
        }

        return initBreak;
    }

    public static boolean isStopCharacter(int codePoint) {
        int label = Character.getType(codePoint);
        
        return (Character.isWhitespace(codePoint) ||
            Character.isSpaceChar(codePoint) ||
            Character.isISOControl(codePoint) ||
            !Character.isValidCodePoint(codePoint) ||
            (label == Character.DASH_PUNCTUATION) ||
            (label == Character.OTHER_PUNCTUATION) ||
            (label == Character.OTHER_SYMBOL) ||
            (label == Character.START_PUNCTUATION) ||
            (label == Character.END_PUNCTUATION));
    }
    
    //public static void main(String[] args) {
      //  System.out.println("ไทย = " + seg.dict.getEntry("ไทย"));
      //  System.out.println("อะไร = " + seg.dict.getEntry("อะไร"));
      //  System.out.println("โแะ = " + seg.dict.getEntry("โแะ"));
      //  System.out.println("v = " + seg.dict.getEntry("v"));
    //}
}
